import json
import pandas as pd
from multiprocessing import Pool
import pymongo
import requests
from get_field_name.config import *
from get_field_name.save_to_db import *
# 连接数据库
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]
table = db['fourth_name']

def get_fourth_field_name_list():
    data = pd.DataFrame(list(table.find()))
    data = data[["id", "n", "first_field_name", "second_field_name", "third_field_name"]]
    fourth_field_name_list = []
    for i in data.index:
        fourth_field_name_list.append({"id": data.loc[i][0], "fourth_name": data.loc[i][1], "first_name": data.loc[i][2], "second_name": data.loc[i][3], "third_name": data.loc[i][4]})
    # for n in third_field_name_list:
    #     print(n)
    return fourth_field_name_list
fourth_field_name_list = get_fourth_field_name_list()

def get_fifth_field_name(url, index):
    try:
        response = requests.get(url, verify=False)
        if response.content:  # 返回成功
            return response.content
    except requests.ConnectionError as e:
        print('url出错', e.args)
        pool = Pool(4)
        pool.map(main, index)


def main(index):
    for i in range(index, len(fourth_field_name_list)):
        url = "https://academic.microsoft.com/api/analytics/topics/hierarchy?topicPath=" + str(
            fourth_field_name_list[i]['id'])
        print("index:", i)
        print(fourth_field_name_list[i]['fourth_name'])
        tansformData = json.loads(get_fifth_field_name(url, i))
        fifth_field_name = tansformData['ct']
        parent_field_name = tansformData['bc']
        # for pp in parent_field_name:
        #     print(pp)
        for ss in fifth_field_name:
            ss['first_field_name'] = fourth_field_name_list[i]['first_name']
            ss['second_field_name'] = fourth_field_name_list[i]['second_name']
            ss['third_field_name'] = fourth_field_name_list[i]['third_name']
            ss['fourth_field_name'] = parent_field_name[0]
        try:
            save_data("fifth_name", fifth_field_name)
        except Exception:
            print("index:", i)


if __name__ == '__main__':
    pool = Pool(4)
    pool.map(main, 2350)